﻿using System.Threading;
using DM4;
using sscore;
using System;
using System.Collections.Generic;
using sscore.Utils;
using System.Text.RegularExpressions;
using System.IO;
using sscore.Requests;

namespace KS
{
    public class KS : ModuleBase
    {
        private readonly string _searchUrl;
        private readonly string _offenderLink;
        private readonly string _imageLink;
        private const string StateName = "KS";
        private ResultFile _imagesResultFile;
        readonly string[] _searchParam = { "A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z" };

        public KS()
        {
            //initialize main system constants
            ModuleCode = "SO_KS";
            Description = "KS Sex Offenders";
            Version = "1.0";

            //for creating sessions
            StartParam = 0;
            StopParam = _searchParam.Length - 1;

            //initialize help 
            AddToHelp(DM4Consts.StartPosition, String.Format("{0}...{1}", StartParam, StopParam));
            AddToHelp(DM4Consts.EndPositon, String.Format("{0}...{1}", StartParam, StopParam));

            AddToHelp(DM4Consts.IsDownload, "true|false");
            AddToHelp(DM4Consts.IsDownloadImg, "true|false");
            AddToHelp(DM4Consts.InsertToDb, "false|true", "(Expected: servername, dbname)");
            AddToHelp(DM4Consts.ServerName, "");
            AddToHelp(DM4Consts.DbName, "");
            AddToHelp(DM4Consts.IsClearDb, "false|true");
            AddToHelp(DM4Consts.TestParam, "false|true");

            RootUrl = "http://www.accesskansas.org/";
            _searchUrl = RootUrl + "kbi/offender_registry/";
            _offenderLink = RootUrl + "kbi/offender_registry/?id={0}";
            _imageLink = RootUrl + "kbi/offender_registry/oimgs/{0}.jpg";
        }

        private void SetParametersDefaults()
        {
            Params.CreateBooleanParam(DM4Consts.IsDownload, true);
            Params.CreateBooleanParam(DM4Consts.IsDownloadImg, true);
            Params.CreateBooleanParam(DM4Consts.InsertToDb, false);
            Params.CreateBooleanParam(DM4Consts.IsClearDb, false);
            Params.CreateStringParam(DM4Consts.DbName, "");
            Params.CreateStringParam(DM4Consts.ServerName, "");
            Params.CreateIntegerParam(DM4Consts.StartPosition, StartParam);
            Params.CreateIntegerParam(DM4Consts.LastPosition, Params.GetIntegerParam(DmConsts.StartPosition));
            Params.CreateIntegerParam(DM4Consts.EndPositon, StopParam);
            Params.CreateIntegerParam(DM4Consts.LastRecord, 0);
        }

        public override OperationResult Init(IDM4Core iCore, DmTaskParams param)
        {
            //use basic initialization sequence
            base.Init(iCore, param);

            SetParametersDefaults();
            return new OperationResult { Result = DM4Consts.Ok };
        }

        public override OperationResult Start()
        {
            DoWork();
            return new OperationResult { Result = DM4Consts.Ok };
        }

        public override OperationResult DoWork()
        {
            OperationResult or = new OperationResult { Result = DM4Consts.ErrorUnknown };

            if (!Ct.IsCancellationRequested)
            {
                WriteLog("Start parameters:", MessageType.Warning);
                foreach (KeyValuePair<string, string> kvp in Params.GetDictionary()) //get all existing parameters
                    WriteLog(String.Format("Key = {0}, Value = {1}", kvp.Key, kvp.Value), MessageType.Warning);

                if (Params.GetBooleanParam(DM4Consts.IsDownload))
                    Load();

                if (Params.GetBooleanParam(DM4Consts.InsertToDb))
                    if ((Params.GetStringParam(DM4Consts.DbName) == "") || (Params.GetStringParam(DM4Consts.ServerName) == ""))
                    {
                        or.Message = String.Format("{0} or {1} is empty", DM4Consts.DbName, DM4Consts.ServerName);
                        ResultCode = DM4Consts.SessionResultCodeError;
                        ResultMessage = or.Message;
                        UpdateStatus(DmConsts.Error);
                        WriteErrorLog(or.Message); //writting to error and session log   

                        return or;
                    }
                    else
                    {
                        //insert offenders to db 
                        UpdateStatus(DM4Consts.SessionInsertToDb);

                        Offenders.OffInsertToDatabase(StateName, Params.GetStringParam(DmConsts.DbName),
                             Params.GetStringParam(DmConsts.ServerName), ResultDir, Params.GetBooleanParam(DmConsts.IsClearDb),
                             WriteLog);

                    }

                //check if need abort task
                if (Ct.IsCancellationRequested)
                {
                    ResultCode = DM4Consts.SessionResultTerminateByUser;
                    ResultMessage = "Session terminated by user";
                    UpdateStatus(DM4Consts.SessionWait);
                    WriteLog("Session aborted by user", MessageType.Cool);
                }
                else
                {
                    UpdateProgress(1);
                    ResultCode = DM4Consts.SessionResultCodeOk;
                    ResultMessage = String.Format("Session complete, errors : {0}, records : {1}", TaskParams.ErrorCount, TaskParams.RecordCount);
                    UpdateStatus(DM4Consts.SessionComplete);
                    WriteLog("------------Finish----------------", MessageType.Cool);
                }
            }
            else
            {
                ResultCode = DM4Consts.SessionResultTerminateByUser;
                ResultMessage = "Session terminated by user";
                UpdateStatus(DM4Consts.SessionWait);
                WriteLog("Session aborted by user", MessageType.Cool);
            }


            return or;
        }

        private void Load()
        {
            string resultFileName = String.Format("{0}\\{1}.{2}", ResultDir, TaskParams.SessionId, DmConsts.ResultFilesExtension);
            bool workFlag = true;

            ResultFile = new ResultFile(resultFileName, OffendersResult.GetHeader());
            _imagesResultFile = new ResultFile(String.Format("{0}\\{1}_images.{2}", ResultDir, TaskParams.SessionId, DM4Consts.ResultFilesExtension), "FileName|FileUrl");
            try
            {
                while (workFlag)
                {
                    //рабочий цикл
                    string error;

                    try
                    {
                        error = "";  //Required!!  /*if (!String.IsNullOrEmpty(error)) --> restart thread; else --> download finished*/

                        WriteLog("Start download...");
                        Searching();
                    }
                    catch (Exception e)
                    {
                        error = e.Message;
                    }

                    if (!String.IsNullOrEmpty(error))
                    {
                        Thread.Sleep(30000);
                        WriteErrorLog(error);
                        WriteLog("Try continue work...", MessageType.Warning);
                    }
                    else
                    {
                        workFlag = false;  //все нормально откачало, заканчиваем работу
                    }
                }
            }
            finally
            {
                ResultFile.Close();
                _imagesResultFile.Close();
            }
        }

        private void GetCookies()
        {
            WriteLog("get cookies");
            string html = GetRequest.ReturnCookies(_searchUrl, Rpm);
            CheckPage(html);
            //PHPSESSID=lob1m0rgsjphcfl2sb2msn1o01; path=/,BIGipServerAPP-01=9zPlgZfbTwbmvX9i2/r4yXnQdKxpymdHRDptScCQy2Ie9eGPb1ehPEcSJKxxwBuKA/i8+lnCSHFBSw==; path=/
            Rpm.Cookies = Rpm.Cookies.Replace("path=/,", "").Replace("path=/", "");
        }

        private void CheckPage(string html)
        {
            if (!RegExpr.IsMatch(html, @"(?si:Kansas Bureau of Investigation)"))
                throw new Exception(DmConsts.ExcNeedRestart + Environment.NewLine + html);
        }

        private void Searching()
        {
            if (Test)
            {
                Params.SetIntegerParam(DmConsts.LastPosition, Params.GetIntegerParam(DmConsts.StartPosition));
                Params.SetIntegerParam(DmConsts.EndPositon, Params.GetIntegerParam(DmConsts.StartPosition));
            }

            if (Params.GetIntegerParam(DmConsts.EndPositon) > StopParam)
                Params.SetIntegerParam(DmConsts.EndPositon, StopParam);

            GetCookies();
            for (int indSearchParam = Params.GetIntegerParam(DmConsts.LastPosition); indSearchParam <= Params.GetIntegerParam(DmConsts.EndPositon); indSearchParam++)
            {
                //check if need abort task
                if (Ct.IsCancellationRequested)
                    break;

                Params.SetIntegerParam(DmConsts.LastPosition, indSearchParam);
                UpdateProgress();

                WriteLog(string.Format("Process search param [{0}][{1}]; page [{2}]",
                    Params.GetIntegerParam(DmConsts.LastPosition),
                    _searchParam[Params.GetIntegerParam(DmConsts.LastPosition)],
                    "1"));

                string html = PostRequest.RefererWithCookies(_searchUrl, _searchUrl,
                         String.Format("fname=&lname={0}&address=&city=&zip=&county=&radio1=3&violent=1&drug=1&sex=1&submit=Submit&page=results&script=true", 
                         _searchParam[Params.GetIntegerParam(DmConsts.LastPosition)])
                     , Rpm);
                CheckPage(html);

                int iPage = Params.GetIntegerParam(DmConsts.LastRecord) / 20 + 1;
                if (iPage > 1)
                {
                    WriteLog(string.Format("go to result page with search param. [{0}][{1}]; page [{2}]",
                        Params.GetIntegerParam(DmConsts.LastPosition),
                        _searchParam[Params.GetIntegerParam(DmConsts.LastPosition)],
                        iPage));

                    html = GetRequest.RefererWithCookies(String.Format("{0}?page=results&pageID={1}", _searchUrl, iPage), _searchUrl, Rpm);
                    CheckPage(html);
                }

                NavigateByPage(html, iPage);
            }
        }

        private void NavigateByPage(string html, int iPage)
        {
            bool isNextPageExists;
            do
            {
                if (Ct.IsCancellationRequested)
                    break;

                CheckPage(html);
                ParseResultPage(html, iPage);
                isNextPageExists = false;

                if (RegExpr.IsMatch(html, @"(?si:<strong>\s*Next\s*</strong>)"))
                {
                    isNextPageExists = true;
                    iPage++;

                    WriteLog(string.Format("go to result page with search param. [{0}][{1}]; page [{2}]",
                        Params.GetIntegerParam(DmConsts.LastPosition),
                        _searchParam[Params.GetIntegerParam(DmConsts.LastPosition)],
                        iPage));

                    html = GetRequest.RefererWithCookies(
                        String.Format("{0}?page=results&pageID={1}", _searchUrl, iPage),
                        String.Format("{0}?page=results&pageID={1}", _searchUrl, (iPage - 1)),
                        Rpm);

                    if (RegExpr.IsMatch(html, @"(?si:You must enter one or more of the following fields)"))
                        throw new Exception(DmConsts.ExcNeedRestart + Environment.NewLine + html);
                }
            } while (isNextPageExists);
            Params.SetIntegerParam(DmConsts.LastRecord, 0);
        }

        private void ParseResultPage(string html, int iPage)
        {
            html = HtmlUtils.KillSpace(HtmlUtils.ReplaceSpecSymbols(html.Replace("\r\n", " ").Replace("\t", " ").Replace("\r", " ").Replace("\n", " ")));

            if (RegExpr.IsMatch(html, @"(?si:No results were found for your search)"))
            {
                WriteLog("No results were found");
                return;
            }

            MatchCollection mc = RegExpr.ParseMatchCollection(html, @"(?si:oid:""([\S]+)"")");
            WriteLog(String.Format("found [{0}] matches", mc.Count));
            if (mc.Count == 0)
                throw new Exception(DmConsts.ExcNeedRestart + Environment.NewLine + html);

            List<string> list = new List<string>();

            foreach (Match matchLoc in mc)
                if (!list.Contains(matchLoc.Groups[1].Value))
                    list.Add(matchLoc.Groups[1].Value);

            WriteLog(String.Format("found [{0}] id[s]", list.Count));

            int iOffender = 20 * (iPage - 1);
            foreach (string lookupKey in list)
            {
                if (Ct.IsCancellationRequested)
                    break;

                if (iOffender >= Params.GetIntegerParam(DmConsts.LastRecord))
                {
                    string offenderHtml = GetRequest.WithCookies(string.Format(_offenderLink, lookupKey), Rpm);
                    WriteLog(String.Format("ParseOffender [{0}][{1}]", Params.GetIntegerParam(DmConsts.LastRecord), lookupKey));
                    CheckPage(offenderHtml);
                    if (!RegExpr.IsMatch(offenderHtml, @"(?si:Registered Offender Search Details)"))
                        throw new Exception("Offenders page:" + DmConsts.ExcNeedRestart);
                    ParseOffender(offenderHtml, lookupKey);

                    UpdateRecordCount(1);
                    Params.SetIntegerParam(DmConsts.LastRecord, Params.GetIntegerParam(DmConsts.LastRecord) + 1);
                }
                iOffender++;
            }
        }

        private void ParseOffender(string offenderHtml, string lookupKey)
        {
            offenderHtml = HtmlUtils.KillSpace(HtmlUtils.ReplaceSpecSymbols(offenderHtml.Replace("\r\n", " ").Replace("\t", " ").Replace("\r", " ").Replace("\n", " ")));
            using (OffendersResult or = new OffendersResult())
            {
                or.Source_State = StateName;
                or.offenderLink = string.Format(_offenderLink, lookupKey);
                or.Source_id = lookupKey;

                string tmp = "";
                or.fullname = HtmlUtils.KillSpace(RegExpr.Parse(offenderHtml, @"(?si:<h3>(.*?)</h3>)")).Trim();

                foreach (Match matchLoc in RegExpr.ParseMatchCollection(RegExpr.Parse(offenderHtml, @"(?si:<div id=""aliastable"">(.*?)</table>)")
                   , "(?si:<tr>(.*?)</tr>)"))
                {
                    if (!matchLoc.Groups[1].Value.Contains("<th>"))
                        tmp = RegExpr.Parse(matchLoc.Groups[1].Value.Trim(), @"(?si:<td>(.*?)</td>)").Trim();
                    if (tmp != "") or.AKA += tmp + ";";
                }

                or.Birthdate = HtmlUtils.KillSpace(RegExpr.Parse(offenderHtml, @"(?si:>DOB</acronym>:(.*?)<br />)")).Trim();
                or.registered = HtmlUtils.KillSpace(RegExpr.Parse(offenderHtml, @"(?si:Registered Since:(.*?)</p>)")).Trim();

                if (offenderHtml.Contains(">NONCOMPLIANT<"))
                    or.Status = "NONCOMPLIANT";
                if (offenderHtml.Contains(">COMPLIANT<"))
                    or.Status = "COMPLIANT";

                or.Title = RegExpr.Parse(offenderHtml, @"(?si:<p class=""offtype"">(.*?)</p>)");

                //Address
                MatchCollection mc = RegExpr.ParseMatchCollection(RegExpr.Parse(offenderHtml, @"(?si:<div id=""primaryaddress"">(.*?)</table>)")
                   , "(?si:<tr>(.*?)</tr>)");
                if (mc.Count > 1)
                {
                    MatchCollection mcTd = RegExpr.ParseMatchCollection(mc[1].Groups[1].Value.Trim(), @"(?si:<td>(.*?)</td>)");
                    try { or.Address_date = mcTd[0].Groups[1].Value.Trim(); }
                    catch { }
                    try { or.Address = mcTd[1].Groups[1].Value.Trim(); }
                    catch { }
                    try { or.City = mcTd[2].Groups[1].Value.Trim(); }
                    catch { }
                    try { or.State = RegExpr.Parse(mcTd[3].Groups[1].Value.Trim(), @"(?si:"">(.*?)</abbr>)"); }
                    catch { }
                    try { or.Zip = mcTd[4].Groups[1].Value.Trim(); }
                    catch { }
                    try { or.County = RegExpr.Parse(mcTd[5].Groups[1].Value.Trim(), @"(?si:"">(.*?)</abbr>)"); }
                    catch { }
                }
                //Employer address
                mc = RegExpr.ParseMatchCollection(RegExpr.Parse(offenderHtml, @"(?si:<div id=""addresstable"">(.*?)</table>)")
                   , "(?si:<tr>(.*?)</tr>)");
                foreach (Match matchLoc in mc)
                    if (matchLoc.Groups[1].Value.Contains("<td>Employer</td>"))
                    {
                        MatchCollection mcTd = RegExpr.ParseMatchCollection(matchLoc.Groups[1].Value.Trim(), @"(?si:<td>(.*?)</td>)");

                        try { or.waddress = mcTd[1].Groups[1].Value.Trim(); }
                        catch { }
                        try { or.Wcity = mcTd[2].Groups[1].Value.Trim(); }
                        catch { }
                        try { or.Wstate = RegExpr.Parse(mcTd[3].Groups[1].Value.Trim(), @"(?si:"">(.*?)</abbr>)"); }
                        catch { }
                        try { or.Wzip = mcTd[4].Groups[1].Value.Trim(); }
                        catch { }
                        try { or.wcounty = RegExpr.Parse(mcTd[5].Groups[1].Value.Trim(), @"(?si:"">(.*?)</abbr>)"); }
                        catch { }
                    }

                //physicaldescription
                mc = RegExpr.ParseMatchCollection(RegExpr.Parse(offenderHtml, @"(?si:<div id=""physicaldescription"">(.*?)</table>)")
                   , "(?si:<tr>(.*?)</tr>)");
                if (mc.Count > 1)
                {
                    MatchCollection mcTd = RegExpr.ParseMatchCollection(mc[1].Groups[1].Value.Trim(), @"(?si:<td>(.*?)</td>)");
                    try { or.height = mcTd[0].Groups[1].Value.Trim(); }
                    catch { }
                    try { or.weight = mcTd[1].Groups[1].Value.Trim(); }
                    catch { }
                    try { or.eye_color = mcTd[2].Groups[1].Value.Trim(); }
                    catch { }
                    try { or.hair_color = mcTd[3].Groups[1].Value.Trim(); }
                    catch { }
                    try { or.Ethnicity = mcTd[4].Groups[1].Value.Trim(); }
                    catch { }
                    try { or.sex = mcTd[5].Groups[1].Value.Trim(); }
                    catch { }
                }

                mc = RegExpr.ParseMatchCollection(RegExpr.Parse(offenderHtml, @"(?si:<div id=""offensetable"">(.*?)</table>)")
                   , "(?si:<tr>(.*?)</tr>)");
                foreach (Match matchLoc in mc)
                    if (!matchLoc.Groups[1].Value.Contains("<th>"))
                    {
                        MatchCollection mcTd = RegExpr.ParseMatchCollection(matchLoc.Groups[1].Value.Trim(), @"(?si:<td>(.*?)</td>)");
                        try { or.Crime += RegExpr.Parse(mcTd[0].Groups[1].Value.Trim(), @"(?si:"">(.*?)</a>)") + ";"; }
                        catch { or.Crime += ";"; }
                        try { or.Crime += mcTd[1].Groups[1].Value.Trim() + ";"; }
                        catch { or.Crime += ";"; }
                        try { or.Crime += RegExpr.Parse(mcTd[2].Groups[1].Value.Trim(), @"(?si:"">(.*?)</abbr>)") + ";"; }
                        catch { or.Crime += ";"; }
                        try { or.Crime += RegExpr.Parse(mcTd[3].Groups[1].Value.Trim(), @"(?si:"">(.*?)</abbr>)") + ";"; }
                        catch { or.Crime += ";"; }
                        try { or.Crime += mcTd[4].Groups[1].Value.Trim() + ";"; }
                        catch { or.Crime += ";"; }
                        try { or.Crime += mcTd[5].Groups[1].Value.Trim() + ";"; }
                        catch { or.Crime += ";"; }
                        try { or.Crime += mcTd[6].Groups[1].Value.Trim() + ";"; }
                        catch { or.Crime += ";"; }
                        try { or.Crime += mcTd[7].Groups[1].Value.Trim() + "~"; }
                        catch { or.Crime += "~"; }
                    }
                if (or.Crime != "") or.Crime = "Offense; City; State; County; Offense Date; Conviction Date; Victim Age; Victim Gender~" + or.Crime;

                mc = RegExpr.ParseMatchCollection(RegExpr.Parse(offenderHtml, @"(?si:<div id=""vehicletable"">(.*?)</table>)")
                   , "(?si:<tr>(.*?)</tr>)");
                foreach (Match matchLoc in mc)
                    if (!matchLoc.Groups[1].Value.Contains("<th>"))
                    {
                        MatchCollection mcTd = RegExpr.ParseMatchCollection(matchLoc.Groups[1].Value.Trim(), @"(?si:<td>(.*?)</td>)");
                        
                        for (int i = 0; i <= 5; i++)
                            try { or.Vehicle += mcTd[i].Groups[1].Value.Trim() + ";"; }
                            catch { or.Vehicle += ";"; }

                        try { or.Vehicle += RegExpr.Parse(mcTd[6].Groups[1].Value.Trim(), @"(?si:"">(.*?)</abbr>)") + "~"; }
                        catch { or.Vehicle += "~"; }
                    }

                if (or.Vehicle == "0;;;;;NONE;~") or.Vehicle = "";
                if (or.Vehicle != "") or.Vehicle = "Year; Make; Model; Style; Color; Plate Number; Plate State~" + or.Vehicle;

                //Tattoos </h4>  /</ul>
                mc = RegExpr.ParseMatchCollection(RegExpr.Parse(offenderHtml, @"(?si:Tattoos </h4>(.*?)</ul>)")
                   , "(?si:<li>(.*?)</li>)");
                foreach (Match matchLoc in mc)
                    or.marks += matchLoc.Groups[1].Value.Trim() + ";";

                if (Params.GetBooleanParam(DmConsts.IsDownloadImg))
                {
                    int iImg = 0;
                    MatchCollection mcImg = RegExpr.ParseMatchCollection(
                        RegExpr.Parse(offenderHtml, @"(?si:<div id=""photos"">(.*?)<div class=""yui-u"">)"),
                        @"(?si:oimgs/([\S]+).jpg)");

                    if (mcImg.Count > 0)
                        foreach (Match m in mcImg)
                        {
                            string imgName;
                            if (iImg == 0)
                            {
                                or.Image_ID = Guid.NewGuid().ToString();
                                or.ImageLink = string.Format(_imageLink, m.Groups[1].Value);
                                imgName = String.Format("{0}\\{1}\\{2}.jpg", ResultDir, DmConsts.PhotosDir, or.Image_ID);
                            }
                            else
                                imgName = String.Format("{0}\\{1}\\{2}_{3}.jpg", ResultDir, DmConsts.PhotosDir, or.Image_ID, iImg);

                            Offenders.DownloadImageWithExceptions(_imagesResultFile, imgName, string.Format(_imageLink, m.Groups[1].Value), TaskParams.SessionId);
                            iImg++;
                        }//foreach (Match m in mcImg)
                }

                if (or.Image_ID != "") or.Image_ID += ".jpg";

                if (or.Image_ID != "" && !File.Exists(String.Format("{0}\\{1}\\{2}", ResultDir, DmConsts.PhotosDir, or.Image_ID)))
                    throw new Exception(DmConsts.ExcImgNotFound);

                ResultFile.Write(or.GetValues());
            }
        }

        private void UpdateProgress()
        {
            if (Params.GetIntegerParam(DmConsts.StartPosition) > Params.GetIntegerParam(DmConsts.EndPositon))
                throw new Exception(String.Format("Invalid parameters {2}[{0}]; {3} [{1}]",
                    Params.GetIntegerParam(DmConsts.StartPosition), Params.GetIntegerParam(DmConsts.EndPositon),
                    DmConsts.StartPosition, DmConsts.EndPositon));
            if (Params.GetIntegerParam(DmConsts.LastPosition) > Params.GetIntegerParam(DmConsts.EndPositon))
                throw new Exception(String.Format("Invalid parameters {2}[{0}]; {3} [{1}]",
                    Params.GetIntegerParam(DmConsts.LastPosition), Params.GetIntegerParam(DmConsts.EndPositon),
                    DmConsts.StartPosition, DmConsts.EndPositon));
            if (Params.GetIntegerParam(DmConsts.StartPosition) > Params.GetIntegerParam(DmConsts.LastPosition))
                throw new Exception(String.Format("Invalid parameters {2}[{0}]; {3} [{1}]",
                    Params.GetIntegerParam(DmConsts.StartPosition), Params.GetIntegerParam(DmConsts.LastPosition),
                    DmConsts.StartPosition, DmConsts.LastPosition));
            UpdateProgress(((double)(Params.GetIntegerParam(DmConsts.LastPosition) - Params.GetIntegerParam(DmConsts.StartPosition))) / (Params.GetIntegerParam(DmConsts.EndPositon) - Params.GetIntegerParam(DmConsts.StartPosition) + 1));
        }
    }
}
